package main

import (
	"spider/common"
	_ "spider/common/spiderlog"
	"spider/engine"
	"spider/fetcher"
	"spider/itemSaver"
	"spider/models"
	"spider/parse"
	"time"
)

var consumerEndChan = make(chan int)
var rateLimiter <-chan time.Time

func main() {
	//解析配置文件
	err := common.InitConfig("./conf.json")
	if nil != err {
		panic(err)
	}
	rateLimiter = time.Tick(1000 * 1000 * (common.G_config.Rate)) //爬取频率限制 Rate为1 表示1ms

	//初始化连接Mysql
	models.Connect()

	var itemsChan chan interface{}
	itemsChan = make(chan interface{})

	//开起消费单条数据
	go itemSaver.ConsumerItem(itemsChan, consumerEndChan)

	var header models.Header
	header = make(models.Header)

	header["Referer"] = "https://bj.meituan.com/meishi/"

	header = common.MergreHeader(header, common.CommonHeader)


	req := models.Request{
		Url:        "https://bj.meituan.com/meishi/",
		Header:     header,
		ReqFunc:    fetcher.FetchGet,
		Parser:     parse.ParseMeituanList,
		ParserName: "ParseMeituanList",
		DataFromParent:models.DataFromParent{"page":1,"cate":""},
	}

	engine.Run(itemsChan, rateLimiter, req)
	close(itemsChan)
	<-consumerEndChan

}
